package com.jobsys.pageprocesser;

import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.jobsys.dbprocess.Dbjob;
import com.jobsys.model.MJobinfo;
import com.jobsys.utils.xx;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class BaiPinPage implements PageProcessor {
	// First part:抓取包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setDomain("zhaopin.baidu.com").setRetryTimes(2).setTimeOut(10000).setSleepTime(5000)
			.addHeader("Connection", "keep-alive")
			.addHeader("User-Agent",
					"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36")
			.addHeader("X-Requested-With", "XMLHttpRequest")
			.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
			.addHeader("accept-encoding", "gzip, deflate, sdch, br").addHeader("Accept-Language", "zh-CN,zh;q=0.8")
			.addHeader("Referer", "http://zhaopin.baidu.com/campus");

	// .addHeader("cookie",
	// "BAIDUID=88B9413D69AEC31E82A4AA08DF9C3784:FG=1; PSTM=1481591372;
	// BIDUPSID=C457916F13ECCAE0B25B1B5A7175A3CB;
	// BDUSS=zE3MFdNUm5ET2ZEck1vdnBsbG5jTEREbWYxWkFyRWtwbDc0RG1qUTNxTzEtQ1JaSVFBQUFBJCQAAAAAAAAAAAEAAACP7JsmeXVhbnhpbmc1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALVr~Vi1a~1YT;
	// URLTITLESALARY=%20%28*%29/szzw%3Fdetailidx%3D0%26city%3D%u5317%u4EAC%26id%3Dhttps%253A%252F%252Fwww.lagou.com%252Fjobs%252F3014517.html%253Futm_source%253Dm_cf_seo_ald_zhw%26query%3D%u62C9%u52FE%u7F51%28cookie%29%3Cspan%3EBD%u7ECF%u7406%uFF08%u5F71%u89C6%u65B9%u5411%uFF09%28%u5317%u4EAC%u8F7B%u5A01%u79D1%u6280%u6709%u9650%u8D23%u4EFB%u516C%u53F8%29%3C/span%3E%28cookie%298000-15000%u5143;
	// Hm_lvt_dc173081ad0848b7d3e412373bb02119=1493863174,1493863678,1493865810;
	// Hm_lpvt_dc173081ad0848b7d3e412373bb02119=1493867987; PSINO=3;
	// H_PS_PSSID=1422_13290_21107_17001_20697_21672_20927;
	// BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BCLID=12217578648271245060;
	// BDSFRCVID=d5FsJeC62wPtHn3ZWZMDudV6ig5vrlbTH6aIOvKM8PiEuRQ3Xv8OEG0PJU8g0Ku-HP3qogKKXgOTHw3P;
	// H_BDCLCKID_SF=tJutVI0XtDvbfP0kh-rqK-FHqxby26POQjneaJ5nJDoNMx3VytTA3T-kXlobLj505bvBLnCaQpP-HqTl-nQi5U4LBNoGttvbfgJHKl0MLPjYbb0xyn_VyhOyKfnMBMnv52OnaP0bLIF5MK_Rj5then-W5gTQa4Qj2C_X3b7EfK82EtO_bfbT2MbyQn6ILpTUJ5n7an6vbbrBqPOa-lJbb5D7KabZqj_jJR-8oCIhf-3bfTr1-tJo5DCShUFsbqvC-2Q-5KL-Jq5PO-T_WltbMxP0bbDqa4bDbm7roMbdJJjzjM7GWqjZKfb-Ltcht4bqJmTxoUJhBCnJhhvGqfTRqtPebPRiXPb9QgbfoMtLtCthbKIlejL35n-WqxvKK-RtHD7yWCv6aqv5OR5Jj6K-QfKn2t6zL-cL-TTb-DQV5-jhEp4R3MA--fFJBpokLqcD3b5C2RLa5-Lhsq0x0bQYe-bQyp_LWq-L2KOMahv95h7xO-0R05CajTcQjatjqb3HKC7-sDKh2J7DKROvhjRiX6kyyxom3bvxt5Tx34QuQbTnfl7sWhO00JQ3WlAO2b3maTAeaDFhJKI2MK0xenOK-tt8qG_DatRaHD7yWCvOQhj5OR5JLn7nDnL1-qjzLhI8fTKHbl5Ya-KMo4P9hnOThbJyyGCDt5FjJnKtVbO8bTr0jbjvK-r_q4tehHR8-jOeWDTm5-nTtbC-E4QL3p62jfDBK-vWJMJf2DTlQIOHJPt5OCFRj5K5j5b0epJf-K6Q2CTjBRu8Kb7VbU5J0bbkbftd2-te0hTIQNQUhq6SBx5CO-5wKnbl-l-Wba5nbTJZfJuH_CP5fC_3fP36q4vEbJ8thmT22-us5nTlQhcH0KLKfUJmy-I5Wl-lXtOtKpKq2CktLtosKfb1MRjVXnjvXjkrMtotQq_eyTFO2h5TtUJkSDnTDMRhqfkrDpQyKMniyIv9-pn5tMt0HPonHj_WjjQ-3H;
	// Hm_lvt_6cbedfe16e204b60185852a9f6fa152f=1493868504;
	// Hm_lpvt_6cbedfe16e204b60185852a9f6fa152f=1493876423")

	private Logger logger = Logger.getLogger(BaiPinPage.class);

	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	@Override
	public void process(Page page) {
		// TODO Auto-generated method stub
		try {
			logger.info("开始处理页面...");
			//

			JSONObject jobj = JSONObject.parseObject(page.getRawText());
			JSONArray arr = jobj.getJSONObject("data").getJSONObject("data").getJSONArray("disp_data");

			SimpleDateFormat st2 = new SimpleDateFormat("yyyy-MM-dd");
			Date date = st2.parse(st2.format(new Date()));
			// System.out.println(page.getRawText());
			for (int i = 0; i < arr.size(); i++) {
				String title = arr.getJSONObject(i).getString("recruittitle").toString();
				String titlelink = arr.getJSONObject(i).getString("joblink").toString();
				String industry = arr.getJSONObject(i).getString("industry").toString();
				String position = "";
				String address = arr.getJSONObject(i).getString("city").toString();
				String company = arr.getJSONObject(i).getString("officialname").toString();
				// String posttime =
				// arr.getJSONObject(i).getString("lastmod").toString();
				String source = arr.getJSONObject(i).getString("source").toString();
				String sourcelink = arr.getJSONObject(i).getString("sourcelink").toString();
				String type = arr.getJSONObject(i).getString("campustype").toString();

				// 存入数据库中
				MJobinfo info = new MJobinfo();
				info.set("source", source);
				info.set("sourcelink", sourcelink);
				info.set("title", title);
				info.set("titlelink", titlelink);
				info.set("position", position);
				info.set("company", company);
				info.set("city", address);
				// info.set("salary", salary);
				info.set("publishtime", date);
				info.set("industry", industry);
				info.set("type", type);

				if (Dbjob.isExist(titlelink)) {
					continue;
				} else {
					Dbjob.insert(info);
				}
			}

			// http://zhaopin.baidu.com/api/xzzwmidasync?query=&recruitcity=&rn=30&pn=0&sort_key=7
			int totalNum = Integer.parseInt(jobj.getJSONObject("data").getJSONObject("data").getString("listNum"));

			String pn = page.getUrl().regex("&pn=(.*?)&").get();
			int curNum = Integer.parseInt(pn) + arr.size();

			if (curNum < totalNum) {
				String nextUrl = "http://zhaopin.baidu.com/api/xzzwmidasync?query=&recruitcity=&rn=30&pn=" + curNum
						+ "&sort_key=7";
				if (!xx.isEmpty(nextUrl)) {
					page.addTargetRequest(nextUrl);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
			logger.info(e.toString());
		}
	}

}
